package com.zouxian.news;
//每日快讯-新闻
//https://www.express.co.uk/news
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;

import com.zouxian.pipeline.MyFilePipeline;
import com.zouxian.pojo.News;
import com.zouxian.util.ListToString;
import com.zouxian.util.StringUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.PhantomJSDownloader;
import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

public class MyProcessor2  implements PageProcessor{
	//配置
	private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
	private static int count=0;
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

	public void process(Page page) {
		//新建一个新闻对象
		News news=new News();
		//https://www.express.co.uk/news/uk/1022373/Brexit-news-Labour-Angela-Rayner-Brexit-referendum-BBC-Newsnight
		//https://www.express.co.uk/news/royal/1022354/Meghan-Markle-Prince-Harry-royal-party-amsterdam-soho-house
		//获取分类列表url
		String resource="https://www.express.co.uk";
		if(page.getUrl().regex(resource).match()){
			List<String> all = page.getHtml().xpath("//*[@class=\"main-nav\"]/li/a/@href").all();
			if(!all.toString().contains("http")){
				List<String> url = StringUtils.addURL(all, resource);
				page.addTargetRequests(url);
			}else{
				page.addTargetRequests(all);
			}
		}
		Boolean flag=page.getUrl().regex("https://www.express.co.uk/[A-Z 0-9 a-z -]+/[A-Z 0-9 a-z -]+/[0-9]+/[A-Z a-z 0-9 -]+").match();
		if(!flag){
			//获取所有满足条件的url
			page.addTargetRequests(page.getHtml().links().regex("https://www.express.co.uk/[A-Z 0-9 a-z -]+/[A-Z 0-9 -]+/[0-9]+/[A-Z 0-9 -]+").all());
			
		}
		//System.err.println(page.getUrl());
		count++;
		//获取url
		String Url=page.getUrl().get();
		news.setUrl(Url);
		//获取标题
		//String title=page.getHtml().xpath("//*[@id=\"article\"]/div[2]/div/div[1]/header/div[3]/div[1]/div/h1/font").get();
		String title=page.getHtml().$("h1[itemprop=headline]","allText").toString();
		
		if(title==null){
			System.out.println("该链接没有标题，不符合要求！");
			page.setSkip(true);
			
		}
		news.setTitle(title);
		//获取关键字
		//String articalKeyWords=page.getHtml().$("div.submeta__section-labels ul li a","allText").all().toString();
		//news.setArticalKeyWords(articalKeyWords);
		//获取日期
		String GetTime=page.getHtml().$("time[datetime]","allText").all().toString();
		news.setGetTime(GetTime);
		//获取正文
		String Txt=page.getHtml().$("div[class='clearfix hR new-style'] div", "allText").all().toString();
		news.setTxt(Txt);
		//获取摘要 
		String Summary=page.getHtml().$("header.clearfix h3", "allText").all().toString();
		news.setSummary(Summary);
		//获取文章类型
		String TxtType=page.getHtml().$("ul#breadCrumb li:nth-child(3)","allText").all().toString();
		news.setTxtType(TxtType);
		//获取作者
		String author=page.getHtml().$("span[itemprop=author]", "allText").all().toString();
		news.setAuthor(author);
		page.putField("title", news.getTitle());
		page.putField("TitleCh", news.getTitleCh());
		page.putField("Title_Score", news.getTitle_Score());
		page.putField("articalKeyWords", news.getArticalKeyWords());
		page.putField("Txt", news.getTxt());
		page.putField("TxtCh", news.getTxtCh());
		page.putField("Txt_Score", news.getTxt_Score());
		page.putField("Url", news.getUrl());
		page.putField("Summary", news.getSummary());
		page.putField("SummaryCh", news.getSummaryCh());
		page.putField("TxtType", news.getTxtType());
		page.putField("GetTime", news.getGetTime());
		page.putField("opicType", news.getOpicType());
		page.putField("Source", news.getSource());
		page.putField("isdown", news.getIsdown());
		page.putField("relateNewsNumber", news.getRelateNewsNumber());
		page.putField("KeyWords", news.getKeyWords());
		page.putField("schoolName", news.getSchoolName());
		page.putField("country", news.getCountry());
		page.putField("Polar", news.getPolar());
		page.putField("author", news.getAuthor());
		page.putField("language", news.getLanguage());
		page.putField("htmlCode", news.getHtmlCode());
		page.putField("semanticValue", news.getSemanticValue());
		page.putField("mediaIntroduction", news.getMediaIntroduction());
		page.putField("zuanfaNum", news.getZuanfaNum());
		page.putField("pinlunNum", news.getPinlunNum());
		page.putField("shouchangNum", news.getSchoolName());
		page.putField("txtSeg", news.getTxtSeg());
	}
	
	//.setDownloader(new SeleniumDownloader("C:\\Users\\zx\\Downloads\\chromedriver.exe"))
	public static void main(String[] args) {
		long startTime, endTime;
		System.out.println("开始爬取...");
		startTime = System.currentTimeMillis();
		Spider.create(new MyProcessor2()).addUrl("https://www.express.co.uk").addPipeline(new ConsolePipeline()).addPipeline(new MyFilePipeline()).thread(5).run();
		endTime = System.currentTimeMillis();
		System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000)
				+ "秒，抓取了" + count + "条记录");
		
	}

}
